/*
 *  DictionaryBasedTokeniser.java
 *
 *  Copyright (c) 2007
 *  Knowledge Information & Data Management Laboratory
 *  Sirindhorn International Institute of Technology
 *  Thammasat University
 * 
 *  Arthit Suriyawongkul
 *  art <at> siit.net
 *
 *  arthit, 18/6/2550
 *
 *  $Id: DictionaryBasedTokeniser.jav 2820 2001-11-14 17:15:43Z oana $
 */

package net.siit.gate;

import net.siit.util.DataFactory;
import org.speedblue.util.Trie;

import gate.*;
import gate.creole.*;
import gate.creole.tokeniser.DefaultTokeniser;
import gate.util.*;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.StringReader;
import java.io.UnsupportedEncodingException;
import java.io.FileNotFoundException;
import java.util.*;
import java.net.URISyntaxException;
import java.net.URL;


/**
 * This class is the implementation of the resource DICTIONARYBASEDTOKENISER.
 */
public class DictionaryBasedTokeniser extends DefaultTokeniser {
    // tokenization strategies
    //static public final int LONGEST_MATCH = 1;
    //static public final int MAXIMUM_MATCH = 2;

    static public final String TOKEN_LABEL_OTHER = "other";
    static public final String TOKEN_LABEL_WORD  = "word";

    // min/max token length (characters) in dictionary,
    // use to limit comparison window
    private int maxLength;
    private int minLength;

    // Dictionary
    private static Trie dict;

    /*
     * this method gets called whenever an object of this
     * class is created either from GATE GUI or if
     * initiated using Factory.createResource() method.
     */
    public Resource init() throws ResourceInstantiationException {
        fireStatusChanged("Creating a tokeniser");
        fireProgressChanged(0);
        
        if ((dictionaryURL == null) || (dictionaryURL.toString().trim().length() == 0)) {
            throw new ResourceInstantiationException("Dictionary URL must be specified.");
        }
        if ((dictionaryEncoding == null) || (dictionaryEncoding.trim().length() == 0)) {
            fireStatusChanged("Unsupported encoding, use UTF-8.");
            dictionaryEncoding = "UTF-8";
        }
        fireProgressChanged(5);
        
        try {
            dict = DataFactory.createTrieDictionary(dictionaryURL, dictionaryEncoding, new DataFactory.LengthValuer());
            maxLength = dict.getMaxLength();
            minLength = dict.getMinLength();
        } catch (URISyntaxException ex) {
            throw new gate.creole.ResourceInstantiationException("Dictionary URL is file not found: ", ex);
        } catch (UnsupportedEncodingException ex) {
            throw new ResourceInstantiationException("Dictionary encoding is not supported: " + dictionaryEncoding, ex);
        } catch (FileNotFoundException ex) {
            throw new gate.creole.ResourceInstantiationException("Not found dictionary at a given URL: ", ex);
        } catch (IOException ex) {
            throw new gate.creole.ResourceInstantiationException(ex);
        }
        fireProgressChanged(100);

        fireStatusChanged("Instantiated");
        fireProcessFinished();
        return this;
    }

    /*
     * this method should provide the actual functionality of the PR
     * (from where the main execution begins). This method
     * gets called when user click on the RUN button in the
     * GATE GUIs application window.
     */
    public void execute() throws ExecutionException {
        fireProgressChanged(0);

        if (document == null) {
            throw new ExecutionException("No document to process!");
        }

        fireStatusChanged("Tokenising " + document.getName() + "...");

        if (annotationSetName != null && annotationSetName.trim().length() == 0) {
            annotationSetName = null;
        }

        AnnotationSet as = (annotationSetName == null) ?
            document.getAnnotations() :
            document.getAnnotations(annotationSetName);

        //System.err.println("AS: " + as.getName() + " (" + as.size() + ")");

        //length of new-line character(s)
        int nllen = 1;
        String nl = (String) document.getFeatures().get("docNewLineType");
        if (nl.equalsIgnoreCase("CRLF") || nl.equalsIgnoreCase("LFCR")) {
            nllen = 2;
        }

        BufferedReader in = new BufferedReader(
                new StringReader(document.getContent().toString()));

        String line = null;
        try {
            int start, end;
            long index = 0;
            String tokenString;
            String annotName;
            while ((line = in.readLine()) != null) {
                int len = line.length();
                //if (len < minLength) continue;

                start = 0;
                end   = start + minLength;  // end will grows from start+minLength to start+maxLength
                int cStart; // candidate's start
                int cEnd;   // candidate's end
              
                while (end <= len) {
                    cStart = -1;
                    cEnd   = -1;
                    while ((end <= len) && (end < start+maxLength)) {
                        // specifically for Thai:
                        // if next char is "following character", don't cut it here
                        if (end < len) {
                            char check = line.charAt(end);
                            if ( (check >= 0x0e30 && check <= 0x0e3a) ||
                                 (check == 0x0e45) ||
                                 (check == 0x0e47 && check <= 0x0e4e) ) {
                                end++;
                                continue;
                            }
                        }
                        
                        if (dict.getEntry(line.substring(start, end)) > -1) {
                            cStart = start;
                            cEnd   = end;
                        }
                        end++;
                    }

                    if (cStart == -1) {
                        // mark nothing
                        start++; // go further next one character                        
                    } else {
                        tokenString = line.substring(cStart, cEnd);

                        annotName = TOKEN_ANNOTATION_TYPE;  // <Token>

                        FeatureMap features = Factory.newFeatureMap();
                        features.put(TOKEN_STRING_FEATURE_NAME, tokenString);  // @string
                        features.put(TOKEN_LENGTH_FEATURE_NAME, tokenString.length());  // @length
                        features.put(TOKEN_KIND_FEATURE_NAME, TOKEN_LABEL_WORD);  // @kind

                        as.add(Long.valueOf(index+cStart), Long.valueOf(index+cEnd), annotName, features);

                        start = cEnd;
                    }

                    end = start + minLength;
                }

                index += len + nllen;
            }
        } catch (Exception ex) {
            throw new ExecutionException(ex);
        } finally {
            // dereference
            try {
                in.close();
                as = null;
            } catch(IOException ex) {
                throw new ExecutionException(ex);
            }
        }

        fireStatusChanged("Tokenisation complete!");
        fireProgressChanged(100);
        fireProcessFinished();
    }

    /* this method is called to reinitialize the resource */
    public void reInit() throws ResourceInstantiationException {
        this.init();
    }

    /*
     * There are two types of parameters
     * 1. Init time parameters values for these parameters need to be
     * provided at the time of initializing a new resource and these values are
     * not supposed to be changed.
     * 2. Runtime parameters - values for these parameters are provided at the time
     * of executing the PR. These are runtime parameters and can be
     * changed before starting the execution
     * (i.e. before you click on the "RUN" button in the GATE GUI)
     * It is must to provide setter and getter methods for every such
     * parameter declared in the creole.xml.
     *
     * for example to set a value for outputAnnotationSetName
     */
    private String annotationSetName;
    private URL dictionaryURL;
    private String dictionaryEncoding;

    // getters, setters
    public String getAnnotationSetName() {
        return annotationSetName;
    }
    public void setAnnotationSetName(String annotationSetName) {
        this.annotationSetName = annotationSetName;
    }

    public URL getDictionaryURL() {
        return dictionaryURL;
    }
    public void setDictionaryURL(URL url) {
        this.dictionaryURL = url;
    }

    public String getDictionaryEncoding() {
        return dictionaryEncoding;
    }
    public void setDictionaryEncoding(String encoding) {
        this.dictionaryEncoding = encoding;
    }

} // class DictionaryBasedTokeniser
